* update of Cavari & Freedman 2018.  
* First run new_roper_to_dta_surveys.R in R
* set directory and run this code in Stata.
* this code combines the two datasets to create the datafile for our models.  


************************************
local i "31103004"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
rename q40 news_gov
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)

keep weight q25c q25d q25f q25g q25i q25p qb31 q50u q50cc q50ee q50hh ///
	q51jj q51ll q51nn q51qq qa126 qa128 qa129 news_gov party partyln COLLEGE ///
	income

foreach var of varlist q25c q25d q25f q25g q25i q25p qb31 ///
	q50u q50cc q50ee q50hh q51jj q51ll q51nn q51qq ///
	qa126 qa128 qa129 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

foreach var of varlist qa128 qa129 news_gov {
replace `var' = . if `var'>4
}

gen survey="`i'"
saveold newsurveys.dta, replace

clear
gen abd_d=.
saveold SNR_additions.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q25c q25d q25f q25g q25i q25p qb31 ///
	q50u q50cc q50ee q50hh q51jj q51ll q51nn q51qq ///
	qa126 qa128 qa129{
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
saveold newsurveyslong.dta, replace

************************************
local i "31114069"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
rename folgov news_gov
rename pvote12a voted
recode voted (1=1) (2=0) (else=.)
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
keep weight q14a q14b q14d q20 news_gov voted party partyln COLLEGE income

foreach var of varlist q14a q14b q14d q20 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

foreach var of varlist news_gov {
replace `var' = . if `var'>4
}

gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q14a q14b q14d q20{
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "31114198"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
keep weight q86b q86c q86e q95 qxa qxb q96f1 party partyln COLLEGE income

foreach var of varlist q86b q86c q86e q95 qxa qxb q96f1 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q86b q86c q86e q95 qxa qxb q96f1 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "31114648"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
keep weight q46 q47 q55 q80 q81 q82 q92 q99 party partyln COLLEGE income

foreach var of varlist q46 q47 q80 q81 q82 q99 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

foreach var of varlist q55 q92 party {
replace `var' = . if `var'>4
}


gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q46 q47 q55 q80 q81 q82 q92 q99 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "31114931"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q51 party partyln COLLEGE income

foreach var of varlist q51 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q51 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "31114967"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q41af1 q41bf1 q41cf1 q41df1 q41ef1 q41gf1 ///
	q41if2 q41jf2 q41kf2 q41mf2 q41nf2 q70 q71 q83 party partyln COLLEGE income

foreach var of varlist q41af1 q41bf1 q41cf1 q41df1 q41ef1 q41gf1 ///
	q41if2 q41jf2 q41kf2 q41mf2 q41nf2{
recode `var' (1=1) (2=3) (3=2) (else=.)
}

foreach var of varlist q70 q71 q83 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q41af1 q41bf1 q41cf1 q41df1 q41ef1 q41gf1 ///
	q41if2 q41jf2 q41kf2 q41mf2 q41nf2 q70 q71 q83 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "31114968"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q52 q68f1 q69f2 q70f1 q71f2 q88 q91f2 party partyln COLLEGE income

foreach var of varlist q52 q68f1 q69f2 q70f1 q71f2 q91f2 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

foreach var of varlist q88 {
replace `var' = . if `var'>4
}

gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q52 q68f1 q69f2 q70f1 q71f2 q88 q91f2 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "31114969"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q61 q65 q73 party partyln COLLEGE income

foreach var of varlist q61 q65 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

recode q73 (1=1) (2=3) (3=2) (else=.)

gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q61 q65 q73 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "31114971"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q5f1 q6f2 party partyln COLLEGE income

foreach var of varlist q5f1 q6f2 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q5f1 q6f2 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "31115014"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
rename campnii news_elections
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q21 q22 q45 q72 q75 q82 party partyln news_elections COLLEGE income

foreach var of varlist q22 q45 q72 q75 q82 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

recode q21 (1=1) (2=3) (3=2) (4=4) (else=.)
foreach var of varlist news_elections {
replace `var' = . if `var'>4
}

gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q21 q22 q45 q72 q75 q82 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "31115228"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q82 q83 q90a q90c q94 party partyln COLLEGE income

foreach var of varlist q82 q83 q90a q90c party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

recode q94 (1=2) (2=1) (3=3) (else=.)

gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q82 q83 q90a q90c q94 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "31115590"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
rename campnii news_elections
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q66b q66c q66df1 q66e q74 q78 q98 party partyln news_elections COLLEGE ///
	income

foreach var of varlist q66b q66c q66df1 q66e q74 q98 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

foreach var of varlist news_elections q78 {
replace `var' = . if `var'>4
}
gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q66b q66c q66df1 q66e q74 q78 q98 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "USPEW2015-02POL"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q11a q11b q11c q13 q66 q70 q75a q75b q75c q78 party partyln COLLEGE ///
	income

foreach var of varlist q11a q11b q11c q13 q66 q70 q75a q75b q75c party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln
recode q78 (1=1) (2=3) (3=2) (else=.)
gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q11a q11b q11c q13 q66 q70 q75a q75b q75c q78 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "USPEW2015-03POL"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q71 party partyln COLLEGE income

foreach var of varlist q71 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q71 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "USPEW2015-05POL"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q24a q24b q34 q36 q38a q40 q44 party partyln COLLEGE income

foreach var of varlist q24a q24b q34 q36 q44 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln
recode q38a (1=2) (2=1) (3=3) (else=.)
foreach var of varlist q40 party {
replace `var' = . if `var'>4
}

gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q24a q24b q34 q36 q38a q40 q44 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "USPEW2015-07POL"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q35 q61 q64 q75 q76 party partyln COLLEGE income

foreach var of varlist q35 q61 q64 q75 q76 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q35 q61 q64 q75 q76 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "USPEW2015-09POL"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q7 q9 q52 q55 q58 q63 q64 party partyln COLLEGE income

foreach var of varlist q7 q9 q52 q55 q58 q63 q64 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q7 q9 q52 q55 q58 q63 q64 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "USPEW2015-12POL"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q27 q30a q30b q30c q45 q81 q84 party partyln COLLEGE income

foreach var of varlist q27 q30a q30b q30c q45 q81 q84 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q27 q30a q30b q30c q45 q81 q84 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "USPEW2015-GOVERNANCE"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
rename q10 news_gov
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q42c q42d q42f q42g q42h q42m q107xf2 q107yf2 party partyln news_gov COLLEGE income

foreach var of varlist q42c q42d q42f q42g q42h q42m q107xf2 q107yf2 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln
foreach var of varlist news_gov{
replace `var' = . if `var'>4
}

gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q42c q42d q42f q42g q42h q42m q107xf2 q107yf2 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "USPEW2016-0326"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q50c q50f q58m q75 q76 q77 q80 q85 q87f1 q88f2 q90 party partyln COLLEGE income

foreach var of varlist q50c q50f q58m q77 q80 q85 q87f1 q88f2 q90 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

foreach var of varlist q75 q76 party {
replace `var' = . if `var'>4
}


gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q50c q50f q58m q75 q76 q77 q80 q85 q87f1 q88f2 q90 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "USPEW2016-0419"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q28 q31 q37 q40 q41a q41b q41c q42 q47 q52 ///
	q64a q64b q64c q73 q75 q76 party partyln COLLEGE income

foreach var of varlist q28 q31 q37 q40 q41a q41b q41c q47 q52 q64a q64b q64c ///
	q73 q75 q76 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

foreach var of varlist q42 {
replace `var' = . if `var'>3
}


gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q28 q31 q37 q40 q41a q41b q41c q42 q47 q52 ///
	q64a q64b q64c q73 q75 q76 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "USPEW2016-0816"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
rename campnii news_elections 
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q56f1 q58 q66b q76a q76b q76c q78 party partyln news_elections COLLEGE income

foreach var of varlist q56f1 q58 q66b q76a q76b q76c q78 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

foreach var of varlist news_elections {
replace `var' = . if `var'>4
}


gen survey="`i'"
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q56f1 q58 q66b q76a q76b q76c q78 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
saveold newsurveyslong.dta, replace

************************************
local i "USPEW2016-1025"
************************************
set more off
clear

use roper_data/`i'/`i'.dta
rename _all, lower
recode income (1/2=1) (3/4=2) (5=3) (6/7=4) (8/9=5) (else=.)
rename campnii news_elections 
recode educ (6/8=1) (1/5=0) (else=.), gen(COLLEGE)
keep weight q90f1 q91f1 q93f1 q95f1 q96f2 q97f1 q100f1 q103f2 ///
	party partyln news_elections COLLEGE income

foreach var of varlist q91f1 q95f1 q96f2 q97f1 q100f1 q103f2 party {
replace `var' = . if `var'>2
}

replace party=1 if partyln==1
replace party=2 if partyln==2
drop partyln

recode q90f1 (1=1) (2=3) (3=2) (else=.)

foreach var of varlist q93f1 news_elections {
replace `var' = . if `var'>4
}


gen survey="`i'"
label define income 1 "1st - Lowest" 2 "2nd" 3 "3rd" 4 "4th" 5 "5th - Highest", replace
label values income income
saveold newsurveys.dta, replace

clear
use newsurveys.dta, clear

foreach q of varlist q90f1 q91f1 q93f1 q95f1 q96f2 q97f1 q100f1 q103f2 {
clear
use newsurveys.dta, clear
keep if survey=="`i'"
quietly esize twosample `q', by(party)
svret r, keep(r(d) r(lb_d) r(ub_d))
rename r_d abd_d
replace abd_d = abs(abd_d)
gen q="`q'"
gen survey = "`i'"
append using SNR_additions.dta
saveold SNR_additions.dta, replace
}

clear
use newsurveys.dta, clear
gen rid = _n
reshape long q, i(rid) j(t) str
label values q y
rename q y
rename t q
append using newsurveyslong.dta
rename COLLEGE COLLEGEMORE
saveold newsurveyslong.dta, replace
erase newsurveys.dta

*************************************
* prepare for modelling
*************************************


clear
import excel using newsurveys.xlsx, sheet(Sheet1) firstrow
keep if Policy==1
keep survey enddate responserate
rename responserate unit
saveold details.dta, replace

clear
import excel using newsurveys.xlsx, sheet(question) firstrow
merge m:1 survey using details.dta
drop _merge
rename macroeconomic economy
saveold details.dta, replace

clear
use SNR_additions.dta, clear
merge 1:1 survey q using details.dta
drop _merge
gen pap_majortopic = .
replace pap_majortopic=1 if economy==1
replace pap_majortopic=2 if civil_rights==1
replace pap_majortopic=8 if energy==1
replace pap_majortopic=9 if immigration==1
replace pap_majortopic=13 if welfare==1
replace pap_majortopic=16 if field==2

gen year=yofd(enddate)
merge m:1 year pap using newcongressd.dta
drop if _merge==2
drop _merge
rename congress d_congress
gen id=_n
replace id=id+1158
rename (r_lb_d r_ub_d) (d_lower d_upper)
drop enddate pap
saveold SNR_additions.dta, replace 

clear
use "$dropbox/Research_Projects/Survey_non_Response/JOP Data/Pew_Models.dta"
keep abd_d d_lower d_upper survey field economy civil_rights energy immigration welfare values unit year d_congress id
append using SNR_additions.dta

saveold SNR_updated2018.dta, replace
